# -*- coding: utf-8 -*-

import os
import re
import sys
reload(sys)
sys.setdefaultencoding('utf8')

# < single tag > remove
# < common tag > value </ common tag> extract value then put it to next loop
# < script > </script> remove

class PageCleaner(object):
    single_tag_list = ['meta', 'link', 'style', 'img', 'input', 'hr', 'br', '!DOCTYPE', '!--']
    exculded_tag_list = ['head', 'script', 'a', 'img', 'button']

    def __init__(self):
        self.html = ''
        self.node_recorder = list()
        self.extract_content = '' 

    def feed_html(self, filename=''):
        with open(filename, 'r') as f:
            self.html = f.read()

    def clean(self):
        content = self.html
        index = 0
        title_content = ''
        title_start = content.find('<title', index)
        if title_start > 0:
            index = title_start
            title_start = content.find('>', index)
            title_end = content.find('</title', index)
            index = title_end
            title_content = content[title_start+1: title_end] 
            index += len('</title>')

        body_start = content.find('<body', index)
        index = body_start
        body_start = content.find('>', index)
        body_end = content.find('</body', index)
        body_content = content[body_start+1:body_end] 

        for single_tag in self.single_tag_list:
            while True:
                tag_start = body_content.find("<%s" % single_tag)
                if tag_start < 0:
                   break 
                tag_end = body_content.find('>', tag_start)
                part1 = body_content[:tag_start]
                part2 = body_content[tag_end:]
                body_content = part1 + part2

        for exculded_tag in self.exculded_tag_list:
            while True:
                tag_len = len(exculded_tag)
                tag_start = body_content.find("<%s" % exculded_tag)
                if tag_start < 0:
                    break
                tag_end = body_content.find(">", tag_start+tag_len) 

                close_tag_start = body_content.find("</%s" % exculded_tag, tag_end)
                close_tag_end = body_content.find(">", close_tag_start+tag_len)  

                part1 = body_content[:tag_start]
                part2 = body_content[close_tag_end+1:]
                body_content = part1 + part2

        while True:
            tag_start = body_content.find('<')
            if tag_start < 0:
                break
            tag_end = body_content.find('>', tag_start)
            part1 = body_content[:tag_start]
            part2 = body_content[tag_end+1:]
            body_content = part1 + part2

        self.extract_content = body_content
        #print self.extract_content


    def get_content(self):
        return self.html

    def output_file(self, filename):
        with open(filename, 'a+') as f:
            f.write(self.extract_content)


if __name__ == '__main__':
    cleaner = PageCleaner()
    cleaner.feed_html('city.html')
    cleaner.clean()
    cleaner.output_file('example.txt')

    #dirlist = os.listdir('./data/');
    #for i in dirlist:
    #    cleaner.feed_html(os.path.join(os.getcwd(), 'data', i))
    #    cleaner.clean()
    #with open(os.path.join('data_out', i+'.txt'), 'a') as f:
    #    f.write(cleaner.extract_content)
